import os
import sys
import requests
import json
import re
from lxml import etree

cwd = os.getcwd()
sys.path.append(cwd)
print(cwd)

from pub_func import DownloadFile, folder_exist, file_write

base_url = 'http://www.yitubao.com/sucai/jiangxue-247650-{}.html'

headers = {
    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
}

if __name__ == '__main__':
    for i in range(1, 20):
        response = requests.get(url=base_url.format(i), headers=headers)
        response.encoding = 'utf8'
        # print(response.text)
        html = etree.HTML(response.text)
        list_img_url = html.xpath('//*[@id="tiles"]/li/div/a[1]/img/@src')
        print(list_img_url, len(list_img_url))
        path = os.path.join('/home/ubuntu/workspace/img' ,'yitubao_8.csv')
        for url_img in list_img_url:
            content = url_img + ',\n'
            file_write(path, content, 'a') 
